# -*- coding:utf-8 -*-
from database.db_handler import MysqlHander
from database.db_business import DbBusiness
from common.my_http import MyHttp
from common.my_baidu import MyBaidu
from common.my_time import MyTime
from common.my_file import MyFile
from common.macro_data import MacroData
import urllib.request
from bs4 import BeautifulSoup
import glob
import jieba
import re
import threading
import hashlib
import urllib.parse
import time
import os


'''
网络数据
'''


class BaiduLianheguo:
    def __init__(self):
        self.macro = MacroData()
        self.business = DbBusiness()
        self.log_file = open('./out/' + os.path.split(__file__)[-1].split(".")[0] + ".log", 'a', encoding = 'utf-8')
        MyFile.wrtie_log(self.log_file, "开始")
        self.all = 0
        self.error = 0

    def __del__(self):
        MyFile.wrtie_log(self.log_file, "all:" + str(self.all) + ",error:" + str(self.error))
        MyFile.wrtie_log(self.log_file, "结束")
        self.log_file.close()

    def get_data(self):
        end_date = MyTime.forward_relative_date(30)
        datas = ["联合国建设和平委员会","国际贸易中心","联合国难民署","联合国国际法院","联合国儿童基金会","联合国贸易和发展会议","联合国开发计划署","联合国资本发展基金","联合国志愿人员","联合国国际药物管制规划署","联合国环境规划署","联合国人类住区规划署","联合国人居署","联合国人口基金","联合国近东巴勒斯坦难民救济和工程处","世界粮食计划署","联合国粮食署","联合国裁军研究所","联合国训练研究所","联合国训研所","联合国区域间犯罪和司法研究所","联合国社会发展研究所","联合国社发研究所","联合国艾滋病规划署","联合国妇女署","联合国大学","联合国系统职员学院","联合国项目事务厅","联合国人力资源管理厅","联合国安理会","联合国反恐怖主义委员会","联合国制裁委员会","联合国赔偿委员会","联合国建设和平委员会","联合国经济及社会理事会","联合国预防犯罪和刑事司法委员会","联合国麻醉药品委员会","联合国人口与发展委员会","联合国科学和技术促进发展委员会","联合国社会发展委员会","联合国妇女地位委员会","联合国可持续发展委员会","联合国统计委员会","联合国森林论坛","非洲经济委员会","欧洲经济委员会","拉丁美洲和加勒比经济委员会","亚洲及太平洋经济社会委员会","西亚经济社会委员会","联合国方案和协调委员会","联合国地名专家组","联合国全球地理空间信息管理专家委员会","国际会计和报告标准政府间专家工作组","联合国发展政策委员会","联合国可持续发展委员会","国际税务合作专家委员会","公共行政专家委员会","国际麻醉品管制局","联合国人口奖委员会","联合国粮食及农业组织","国际民用航空组织","国际农业发展基金","国际劳工组织","国际海事组织","国际货币基金组织","国际电信联盟","联合国教科文组织","联合国工业发展组织","万国邮政联盟","世界银行","国际复兴开发银行","国际投资争端解决中心","国际开发协会","国际金融公司","多边投资保证机构","世界卫生组织","世界知识产权组织","世界气象组织","世界旅游组织","国际原子能机构","全面禁止核试验条约组织筹备委员会","禁止化学武器组织","世界贸易组织","联合国科学技术组织","残疾人权利公约","联合国防治荒漠化公约","联合国气候变化框架公约","联合国信托基金","联合国民主基金","联合国国际伙伴关系基金","联合国内罗毕办事处","联合国日内瓦办事处","联合国维也纳办事处"]
        #check_title = [word]
        for d in datas:
            next_page = True
            words = [d]
            for i in range(1,5):
                #time.sleep(1)
                if next_page == False:
                    break
                url = MyBaidu.get_url_order_by_time(words, i)
                self.all = self.all + 1
                try:
                    proxy = self.business.query_proxy()
                    res = MyBaidu.get_baidu_data_by_proxy(url, proxy)
                    for r in res:
                        data_date = MyBaidu.calc_date(r["date"])
                        if data_date < end_date:
                            next_page = False
                            break
                        if MyBaidu.check_site(r["site"]) == False:
                            continue
                        if MyBaidu.check_title(r["title"], [[d]]) == False:
                            continue
                        data_title = r["title"]
                        data_site = r["site"]
                        data_url = r["url"]
                        
                        print(r["date"] + data_date)
                        self.macro.add_macro_data(data_date, 180, data_title, data_url, data_site)
                except Exception as e:
                    #i = i - 1
                    print(str(e))
                    self.error = self.error + 1
                    pass

def lianheguo_api():
    d = BaiduLianheguo()
    d.get_data()


if __name__ == '__main__':
    lianheguo_api()
